package cn.edu.dutir.lucene;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Date;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.demo.html.HTMLParser;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;

public class LuceneIndexBuilder {

	static final File INDEX_DIR = new File("lucene_index");

	public static void main(String args[]) throws InterruptedException {
		String usage = "java cn.edu.dutir.unit.lucene.LuceneIndexBuilder <root_directory>";
		if (args.length == 0) {
			System.err.println("Usage: " + usage);
			System.exit(1);
		}

		// if (INDEX_DIR.exists()) {
		// System.out.println("Cannot save index to '" + INDEX_DIR
		// + "' directory, please delete it first");
		// System.exit(1);
		// }

		final File docDir = new File(args[0]);
		if (!docDir.exists() || !docDir.canRead()) {
			System.out
					.println("Document directory '"
							+ docDir.getAbsolutePath()
							+ "' does not exist or is not readable, please check the path");
			System.exit(1);
		}

		Date start = new Date();
		try {
			IndexWriter writer = new IndexWriter(INDEX_DIR,
					new StandardAnalyzer(), true);
			System.out.println("Indexing to directory '" + INDEX_DIR + "'...");
			indexDocs(writer, docDir);
			System.out.println("Optimizing...");
			writer.optimize();
			writer.close();

			Date end = new Date();
			System.out.println(end.getTime() - start.getTime()
					+ " total milliseconds");

		} catch (IOException e) {
			System.out.println(" caught a " + e.getClass()
					+ "\n with message: " + e.getMessage());
		}
	}

	static void indexDocs(IndexWriter writer, File file) throws IOException,
			InterruptedException {
		// do not try to index files that cannot be read
		if (file.canRead()) {
			if (file.isDirectory()) {
				String[] files = file.list();
				// an IO error could occur
				if (files != null) {
					for (int i = 0; i < files.length; i++) {
						indexDocs(writer, new File(file, files[i]));
					}
				}
			} else {
				try {
					Document doc = null;
					if(file.getName().endsWith(".txt")){
						doc = getDocument(file);
					}else if(file.getName().endsWith(".html") || file.getName().endsWith("htm")){
						doc = getHTMLDocument(file);
					}
					if (doc != null) {
						System.out.println("adding " + file);
						writer.addDocument(doc);
					}
				}
				// at least on windows, some temporary files raise this
				// exception with an "access denied" message
				// checking if the file can be read doesn't help
				catch (FileNotFoundException fnfe) {
					;
				}
			}
		}
	}

	public static Document getDocument(File f)
			throws IOException {
		if (!f.getName().endsWith(".txt")) {
			return null;
		}
		Document doc = new Document();

		doc.add(new Field("path", f.getPath(), Field.Store.YES,
				Field.Index.UN_TOKENIZED));
		doc.add(new Field("modified", DateTools.timeToString(f.lastModified(),
				DateTools.Resolution.MINUTE), Field.Store.YES,
				Field.Index.UN_TOKENIZED));
		doc.add(new Field("contents", readText(new FileReader(f)), Field.Store.YES,
				Field.Index.TOKENIZED));

		return doc;
	}

	public static String readText(Reader reader) throws IOException {
		StringBuffer sb = new StringBuffer();
		int len = 0;
		char cbuf[] = new char[1024 * 10];
		while ((len = reader.read(cbuf)) > 0) {
			sb.append(cbuf, 0, len);
		}
		return sb.toString();
	}
	
	public static Document getHTMLDocument(File f) throws IOException,
			InterruptedException {
		if (!f.getName().endsWith(".html")) {
			return null;
		}
		Document doc = new Document();
		char dirSep = System.getProperty("file.separator").charAt(0);

		doc.add(new Field("path", f.getPath().replace(dirSep, '/'),
				Field.Store.YES, Field.Index.UN_TOKENIZED));

		doc.add(new Field("modified", DateTools.timeToString(f.lastModified(),
				DateTools.Resolution.MINUTE), Field.Store.YES,
				Field.Index.UN_TOKENIZED));

		FileInputStream fis = new FileInputStream(f);
		HTMLParser parser = new HTMLParser(fis);

		doc.add(new Field("contents", parser.getReader()));

		doc.add(new Field("summary", parser.getSummary(), Field.Store.YES,
				Field.Index.NO));

		doc.add(new Field("title", parser.getTitle(), Field.Store.YES,
				Field.Index.TOKENIZED));
		return doc;
	}
}
